{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Keras高层接口\n",
    "\n",
    "## 常见功能模块\n",
    "\n",
    "### 常见网络层类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "# 导入 keras 模型，不能使用 import keras，它导入的是标准的 Keras 库\n",
    "from tensorflow import keras\n",
    "# 导入常见网络层类 \n",
    "from tensorflow.keras import layers,Sequential,losses,optimizers,datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=2, shape=(3,), dtype=float32, numpy=array([0.6590012, 0.242433 , 0.0985659], dtype=float32)>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 创建输入张量\n",
    "x = tf.constant([2.,1.,0.1]) \n",
    "# 创建 Softmax 层\n",
    "layer = layers.Softmax(axis=-1) \n",
    "# 调用 softmax 前向计算，输出为 out\n",
    "out = layer(x)\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=3, shape=(3,), dtype=float32, numpy=array([0.6590012, 0.242433 , 0.0985659], dtype=float32)>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 调用 softmax 函数完成前向计算\n",
    "out = tf.nn.softmax(x) \n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 网络容器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: id=63, shape=(4, 2), dtype=float32, numpy=\n",
       "array([[0.        , 0.        ],\n",
       "       [0.0096464 , 0.03690726],\n",
       "       [0.09050844, 0.1508593 ],\n",
       "       [0.9066329 , 1.3231909 ]], dtype=float32)>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 导入 Sequential 容器\n",
    "from tensorflow.keras import layers, Sequential\n",
    "network = Sequential([ # 封装为一个网络\n",
    "    layers.Dense(3, activation=None), # 全连接层，此处不使用激活函数\n",
    "    layers.ReLU(),#激活函数层\n",
    "    layers.Dense(2, activation=None), # 全连接层，此处不使用激活函数\n",
    "    layers.ReLU() #激活函数层\n",
    "])\n",
    "x = tf.random.normal([4,3])\n",
    "# 输入从第一层开始， 逐层传播至输出层，并返回输出层的输出\n",
    "out = network(x) \n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_2 (Dense)              multiple                  15        \n",
      "_________________________________________________________________\n",
      "re_lu_2 (ReLU)               multiple                  0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              multiple                  12        \n",
      "_________________________________________________________________\n",
      "re_lu_3 (ReLU)               multiple                  0         \n",
      "=================================================================\n",
      "Total params: 27\n",
      "Trainable params: 27\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 堆叠 2 次\n",
    "layers_num = 2 \n",
    "# 先创建空的网络容器\n",
    "network = Sequential([]) \n",
    "for _ in range(layers_num):\n",
    "    # 添加全连接层\n",
    "    network.add(layers.Dense(3)) \n",
    "    # 添加激活函数层\n",
    "    network.add(layers.ReLU())\n",
    "# 创建网络参数\n",
    "network.build(input_shape=(4, 4)) \n",
    "network.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "当我们通过 Sequential 容量封装多个网络层时， 每层的参数列表将会自动并入Sequential 容器的参数列表中，不需要人为合并网络参数列表，这也是 Sequential 容器的便捷之处。 Sequential 对象的 trainable_variables 和 variables 包含了所有层的待优化张量列表和全部张量列表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dense_2/kernel:0 (4, 3)\n",
      "dense_2/bias:0 (3,)\n",
      "dense_3/kernel:0 (3, 3)\n",
      "dense_3/bias:0 (3,)\n"
     ]
    }
   ],
   "source": [
    "# 打印网络的待优化参数名与 shape\n",
    "for p in network.trainable_variables:\n",
    "    # 参数名和形状\n",
    "    print(p.name, p.shape) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型装配、 训练与测试\n",
    "\n",
    "### 模型装配"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten (Flatten)            (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 256)               200960    \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 10)                330       \n",
      "=================================================================\n",
      "Total params: 244,522\n",
      "Trainable params: 244,522\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers,Sequential,losses,optimizers,datasets\n",
    "\n",
    "# 创建 5 层的全连接网络\n",
    "network = Sequential([layers.Flatten(input_shape=(28,28)),\n",
    "                      layers.Dense(256, activation='relu'),\n",
    "                      layers.Dense(128, activation='relu'),\n",
    "                      layers.Dense(64, activation='relu'),\n",
    "                      layers.Dense(32, activation='relu'),\n",
    "                      layers.Dense(10, activation='softmax')])\n",
    "network.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模型装配\n",
    "# 采用 Adam 优化器，学习率为 0.01;采用交叉熵损失函数，包含 Softmax\n",
    "# kears sparse_categorical_crossentropy说明：\n",
    "# from_logits=False，output为经过softmax输出的概率值。\n",
    "# from_logits=True，output为经过网络直接输出的 logits张量。\n",
    "network.compile(optimizer=optimizers.Adam(learning_rate=0.01),\n",
    "    loss=losses.CategoricalCrossentropy(from_logits=False),\n",
    "    metrics=['accuracy'] # 设置测量指标为准确率\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(x, y):\n",
    "    # [b, 28, 28], [b]\n",
    "    x = tf.cast(x, dtype=tf.float32) / 255.\n",
    "    y = tf.cast(y, dtype=tf.int32)\n",
    "    y = tf.one_hot(y, depth=10)\n",
    "\n",
    "    return x, y\n",
    "\n",
    "(x, y), (x_test, y_test) = datasets.mnist.load_data()\n",
    "\n",
    "batchsz = 512\n",
    "train_db = tf.data.Dataset.from_tensor_slices((x, y))\n",
    "train_db = train_db.shuffle(1000).map(preprocess).batch(batchsz)\n",
    "\n",
    "test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
    "test_db = test_db.shuffle(1000).map(preprocess).batch(batchsz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "118/118 [==============================] - 8s 66ms/step - loss: 0.3623 - accuracy: 0.8857\n",
      "Epoch 2/5\n",
      "118/118 [==============================] - 7s 62ms/step - loss: 0.1241 - accuracy: 0.9628 - val_loss: 0.1353 - val_accuracy: 0.9586\n",
      "Epoch 3/5\n",
      "118/118 [==============================] - 6s 54ms/step - loss: 0.0901 - accuracy: 0.9731\n",
      "Epoch 4/5\n",
      "118/118 [==============================] - 7s 57ms/step - loss: 0.0749 - accuracy: 0.9781 - val_loss: 0.1384 - val_accuracy: 0.9589\n",
      "Epoch 5/5\n",
      "118/118 [==============================] - 6s 49ms/step - loss: 0.0656 - accuracy: 0.9803\n"
     ]
    }
   ],
   "source": [
    "# 指定训练集为 train_db，验证集为 val_db,训练 5 个 epochs，每 2 个 epoch 验证一次\n",
    "# 返回训练轨迹信息保存在 history 对象中\n",
    "history = network.fit(train_db, epochs=5, validation_data=test_db, validation_freq=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': [0.8857167, 0.96283334, 0.9730667, 0.9780833, 0.98025],\n",
       " 'loss': [0.3642711291193962,\n",
       "  0.12393189992457629,\n",
       "  0.09001679726640384,\n",
       "  0.07489063748518626,\n",
       "  0.06552658499876658],\n",
       " 'val_accuracy': [0.9586, 0.9589],\n",
       " 'val_loss': [0.13528729770332576, 0.13842141330242158]}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 打印训练记录\n",
    "history.history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predict x: (512, 28, 28)\n",
      "[[9.9968231e-01 2.7021329e-07 1.0419684e-04 ... 4.9376224e-05\n",
      "  2.4516762e-06 2.5791560e-05]\n",
      " [1.3073126e-03 6.9929543e-03 5.3222227e-01 ... 4.2156115e-01\n",
      "  5.5427975e-03 2.8291810e-03]\n",
      " [5.7274467e-12 2.3022945e-08 1.7924759e-07 ... 2.1845930e-08\n",
      "  5.2079724e-08 3.8408632e-07]\n",
      " ...\n",
      " [4.5035553e-10 6.9096834e-10 5.1212162e-10 ... 4.3533861e-11\n",
      "  3.0087224e-06 2.7957947e-05]\n",
      " [8.6385694e-07 3.0842818e-06 4.6176160e-06 ... 1.8632505e-05\n",
      "  5.4410077e-07 3.9405664e-04]\n",
      " [2.0788873e-10 9.9992716e-01 7.8617535e-07 ... 4.7528019e-06\n",
      "  3.0525615e-07 3.6315996e-06]]\n"
     ]
    }
   ],
   "source": [
    "# 加载一个 batch 的测试数据\n",
    "x,y = next(iter(test_db))\n",
    "print('predict x:', x.shape) # 打印当前 batch 的形状\n",
    "out = network.predict(x) # 模型预测，预测结果保存在 out 中\n",
    "print(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20/20 [==============================] - 1s 46ms/step - loss: 0.0977 - accuracy: 0.9723\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.09765999196097255, 0.9723]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 模型测试，测试在 db_test 上的性能表现\n",
    "network.evaluate(test_db) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  模型保存与加载\n",
    "\n",
    "### 张量方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved weights.\n"
     ]
    }
   ],
   "source": [
    "bakup_network = network\n",
    "\n",
    "# 保存模型参数到文件上\n",
    "network.save_weights('weights.ckpt')\n",
    "print('saved weights.')\n",
    "del network # 删除网络对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded weights!\n"
     ]
    }
   ],
   "source": [
    "# 重新创建相同的网络结构\n",
    "network = Sequential([layers.Dense(256, activation='relu'),\n",
    "                    layers.Dense(128, activation='relu'),\n",
    "                    layers.Dense(64, activation='relu'),\n",
    "                    layers.Dense(32, activation='relu'),\n",
    "                    layers.Dense(10, activation='softmax')])\n",
    "network.compile(optimizer=optimizers.Adam(lr=0.01),\n",
    "    loss=tf.losses.CategoricalCrossentropy(from_logits=True),\n",
    "    metrics=['accuracy']\n",
    ")\n",
    "# 从参数文件中读取数据并写入当前网络\n",
    "network.load_weights('weights.ckpt')\n",
    "print('loaded weights!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 网络方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saved total model.\n"
     ]
    }
   ],
   "source": [
    "# 保存模型结构与模型参数到文件\n",
    "bakup_network.save('model.h5')\n",
    "print('saved total model.')\n",
    "del network # 删除网络对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-3\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-4\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-0.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-0.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-1.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-1.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-2.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-2.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-3.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-3.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-4.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).layer_with_weights-4.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.iter\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_1\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_2\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.decay\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.learning_rate\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-0.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-1.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-2.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-3.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-3.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-4.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'm' for (root).layer_with_weights-4.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-0.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-1.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-2.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-3.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-3.bias\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-4.kernel\n",
      "WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer's state 'v' for (root).layer_with_weights-4.bias\n",
      "WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/alpha/guide/checkpoints#loading_mechanics for details.\n"
     ]
    }
   ],
   "source": [
    "# 从文件恢复网络结构与网络参数\n",
    "network = keras.models.load_model('model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SavedModel 方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From D:\\MyPythonWork\\deeplearning-with-tensorflow-notes\\venv\\lib\\site-packages\\tensorflow_core\\python\\ops\\resource_variable_ops.py:1781: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Assets written to: model-savedmodel\\assets\n",
      "saving savedmodel.\n"
     ]
    }
   ],
   "source": [
    "# 保存模型结构与模型参数到文件\n",
    "tf.saved_model.save(bakup_network, 'model-savedmodel')\n",
    "print('saving savedmodel.')\n",
    "del network # 删除网络对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load savedmodel from file.\n",
      "Test Accuracy:0.972300\n"
     ]
    }
   ],
   "source": [
    "print('load savedmodel from file.')\n",
    "# 从文件恢复网络结构与网络参数\n",
    "network = tf.saved_model.load('model-savedmodel')\n",
    "# 准确率计量器\n",
    "acc_meter = tf.metrics.CategoricalAccuracy()\n",
    "for x,y in test_db: # 遍历测试集\n",
    "    pred = network(x) # 前向计算\n",
    "    acc_meter.update_state(y_true=y, y_pred=pred) # 更新准确率统计\n",
    "# 打印准确率\n",
    "print(\"Test Accuracy:%f\" % acc_meter.result())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 自定义网络\n",
    "\n",
    "### 自定义网络层\n",
    "\n",
    "首先创建类，并继承自 Layer 基类。 创建初始化方法，并调用母类的初始化函数， 由于是全连接层， 因此需要设置两个参数：输入特征的长度 inp_dim 和输出特征的长度outp_dim，并通过 self.add_variable(name, shape)创建 shape 大小，名字为 name 的张量$W$，并设置为需要优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDense(layers.Layer):\n",
    "    # 自定义网络层\n",
    "    def __init__(self, inp_dim, outp_dim):\n",
    "        super(MyDense, self).__init__()\n",
    "        # 创建权值张量并添加到类管理列表中，设置为需要优化\n",
    "        self.kernel = self.add_weight('w', [inp_dim, outp_dim], trainable=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([<tf.Variable 'w:0' shape=(4, 3) dtype=float32, numpy=\n",
       "  array([[-0.49712783,  0.41782534,  0.34388518],\n",
       "         [-0.28433156, -0.3590107 , -0.27746367],\n",
       "         [ 0.27090347, -0.32440424, -0.2171961 ],\n",
       "         [ 0.4266827 ,  0.3612796 ,  0.8776803 ]], dtype=float32)>],\n",
       " [<tf.Variable 'w:0' shape=(4, 3) dtype=float32, numpy=\n",
       "  array([[-0.49712783,  0.41782534,  0.34388518],\n",
       "         [-0.28433156, -0.3590107 , -0.27746367],\n",
       "         [ 0.27090347, -0.32440424, -0.2171961 ],\n",
       "         [ 0.4266827 ,  0.3612796 ,  0.8776803 ]], dtype=float32)>])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 创建输入为 4，输出为 3 节点的自定义层\n",
    "net = MyDense(4,3) \n",
    "# 查看自定义层的参数列表\n",
    "net.variables,net.trainable_variables "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDense(layers.Layer):\n",
    "    # 自定义网络层\n",
    "    def __init__(self, inp_dim, outp_dim):\n",
    "        super(MyDense, self).__init__()\n",
    "        # 创建权值张量并添加到类管理列表中，设置为需要优化\n",
    "        self.kernel = tf.Variable(tf.random.normal([inp_dim, outp_dim]), trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([<tf.Variable 'Variable:0' shape=(4, 3) dtype=float32, numpy=\n",
       "  array([[ 1.155194  , -2.6825984 ,  0.5081635 ],\n",
       "         [ 1.0916729 ,  1.8726718 ,  0.53675693],\n",
       "         [-1.9162103 ,  0.08422428,  0.47484112],\n",
       "         [ 2.5061731 , -0.28466296, -0.98096603]], dtype=float32)>], [])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 创建输入为 4，输出为 3 节点的自定义层\n",
    "net = MyDense(4,3) \n",
    "# 查看自定义层的参数列表\n",
    "net.variables,net.trainable_variables "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "完成自定义类的初始化工作后，我们来设计自定义类的前向运算逻辑，对于这个例子，只需要完成$O=X@W$矩阵运算，并通过固定的 ReLU 激活函数即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyDense(layers.Layer):\n",
    "    # 自定义网络层\n",
    "    def __init__(self, inp_dim, outp_dim):\n",
    "        super(MyDense, self).__init__()\n",
    "        # 创建权值张量并添加到类管理列表中，设置为需要优化\n",
    "        self.kernel = self.add_weight('w', [inp_dim, outp_dim], trainable=True)\n",
    "\n",
    "    def call(self, inputs, training=None):\n",
    "        # 实现自定义类的前向计算逻辑\n",
    "        # X@W\n",
    "        out = inputs @ self.kernel\n",
    "        # 执行激活函数运算\n",
    "        out = tf.nn.relu(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 自定义网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "my_dense_2 (MyDense)         multiple                  200704    \n",
      "_________________________________________________________________\n",
      "my_dense_3 (MyDense)         multiple                  32768     \n",
      "_________________________________________________________________\n",
      "my_dense_4 (MyDense)         multiple                  8192      \n",
      "_________________________________________________________________\n",
      "my_dense_5 (MyDense)         multiple                  2048      \n",
      "_________________________________________________________________\n",
      "my_dense_6 (MyDense)         multiple                  320       \n",
      "=================================================================\n",
      "Total params: 244,032\n",
      "Trainable params: 244,032\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "network = Sequential([MyDense(784, 256), # 使用自定义的层\n",
    "            MyDense(256, 128),\n",
    "            MyDense(128, 64),\n",
    "            MyDense(64, 32),\n",
    "            MyDense(32, 10)])\n",
    "network.build(input_shape=(None, 28*28))\n",
    "network.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "创建自定义网络类，首先创建类， 并继承自 Model 基类，分别创建对应的网络层对象"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyModel(keras.Model):\n",
    "    # 自定义网络类，继承自 Model 基类\n",
    "    def __init__(self):\n",
    "        super(MyModel, self).__init__()\n",
    "        # 完成网络内需要的网络层的创建工作\n",
    "        self.fc1 = MyDense(28*28, 256)\n",
    "        self.fc2 = MyDense(256, 128)\n",
    "        self.fc3 = MyDense(128, 64)\n",
    "        self.fc4 = MyDense(64, 32)\n",
    "        self.fc5 = MyDense(32, 10)\n",
    "        \n",
    "    def call(self, inputs, training=None):\n",
    "        # 自定义前向运算逻辑\n",
    "        x = self.fc1(inputs)\n",
    "        x = self.fc2(x)\n",
    "        x = self.fc3(x)\n",
    "        x = self.fc4(x)\n",
    "        x = self.fc5(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型乐园"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"resnet50\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, None, None,  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)       (None, None, None, 3 0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1_conv (Conv2D)             (None, None, None, 6 9472        conv1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv1_bn (BatchNormalization)   (None, None, None, 6 256         conv1_conv[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv1_relu (Activation)         (None, None, None, 6 0           conv1_bn[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "pool1_pad (ZeroPadding2D)       (None, None, None, 6 0           conv1_relu[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "pool1_pool (MaxPooling2D)       (None, None, None, 6 0           pool1_pad[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_1_conv (Conv2D)    (None, None, None, 6 4160        pool1_pool[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_1_bn (BatchNormali (None, None, None, 6 256         conv2_block1_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_1_relu (Activation (None, None, None, 6 0           conv2_block1_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_2_conv (Conv2D)    (None, None, None, 6 36928       conv2_block1_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_2_bn (BatchNormali (None, None, None, 6 256         conv2_block1_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_2_relu (Activation (None, None, None, 6 0           conv2_block1_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_0_conv (Conv2D)    (None, None, None, 2 16640       pool1_pool[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_3_conv (Conv2D)    (None, None, None, 2 16640       conv2_block1_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_0_bn (BatchNormali (None, None, None, 2 1024        conv2_block1_0_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_3_bn (BatchNormali (None, None, None, 2 1024        conv2_block1_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_add (Add)          (None, None, None, 2 0           conv2_block1_0_bn[0][0]          \n",
      "                                                                 conv2_block1_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block1_out (Activation)   (None, None, None, 2 0           conv2_block1_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_1_conv (Conv2D)    (None, None, None, 6 16448       conv2_block1_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_1_bn (BatchNormali (None, None, None, 6 256         conv2_block2_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_1_relu (Activation (None, None, None, 6 0           conv2_block2_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_2_conv (Conv2D)    (None, None, None, 6 36928       conv2_block2_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_2_bn (BatchNormali (None, None, None, 6 256         conv2_block2_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_2_relu (Activation (None, None, None, 6 0           conv2_block2_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_3_conv (Conv2D)    (None, None, None, 2 16640       conv2_block2_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_3_bn (BatchNormali (None, None, None, 2 1024        conv2_block2_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_add (Add)          (None, None, None, 2 0           conv2_block1_out[0][0]           \n",
      "                                                                 conv2_block2_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block2_out (Activation)   (None, None, None, 2 0           conv2_block2_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_1_conv (Conv2D)    (None, None, None, 6 16448       conv2_block2_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_1_bn (BatchNormali (None, None, None, 6 256         conv2_block3_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_1_relu (Activation (None, None, None, 6 0           conv2_block3_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_2_conv (Conv2D)    (None, None, None, 6 36928       conv2_block3_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_2_bn (BatchNormali (None, None, None, 6 256         conv2_block3_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_2_relu (Activation (None, None, None, 6 0           conv2_block3_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_3_conv (Conv2D)    (None, None, None, 2 16640       conv2_block3_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_3_bn (BatchNormali (None, None, None, 2 1024        conv2_block3_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_add (Add)          (None, None, None, 2 0           conv2_block2_out[0][0]           \n",
      "                                                                 conv2_block3_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv2_block3_out (Activation)   (None, None, None, 2 0           conv2_block3_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_1_conv (Conv2D)    (None, None, None, 1 32896       conv2_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_1_bn (BatchNormali (None, None, None, 1 512         conv3_block1_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_1_relu (Activation (None, None, None, 1 0           conv3_block1_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_2_conv (Conv2D)    (None, None, None, 1 147584      conv3_block1_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_2_bn (BatchNormali (None, None, None, 1 512         conv3_block1_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_2_relu (Activation (None, None, None, 1 0           conv3_block1_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_0_conv (Conv2D)    (None, None, None, 5 131584      conv2_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_3_conv (Conv2D)    (None, None, None, 5 66048       conv3_block1_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_0_bn (BatchNormali (None, None, None, 5 2048        conv3_block1_0_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_3_bn (BatchNormali (None, None, None, 5 2048        conv3_block1_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_add (Add)          (None, None, None, 5 0           conv3_block1_0_bn[0][0]          \n",
      "                                                                 conv3_block1_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block1_out (Activation)   (None, None, None, 5 0           conv3_block1_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_1_conv (Conv2D)    (None, None, None, 1 65664       conv3_block1_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_1_bn (BatchNormali (None, None, None, 1 512         conv3_block2_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_1_relu (Activation (None, None, None, 1 0           conv3_block2_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_2_conv (Conv2D)    (None, None, None, 1 147584      conv3_block2_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_2_bn (BatchNormali (None, None, None, 1 512         conv3_block2_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_2_relu (Activation (None, None, None, 1 0           conv3_block2_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_3_conv (Conv2D)    (None, None, None, 5 66048       conv3_block2_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_3_bn (BatchNormali (None, None, None, 5 2048        conv3_block2_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_add (Add)          (None, None, None, 5 0           conv3_block1_out[0][0]           \n",
      "                                                                 conv3_block2_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block2_out (Activation)   (None, None, None, 5 0           conv3_block2_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_1_conv (Conv2D)    (None, None, None, 1 65664       conv3_block2_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_1_bn (BatchNormali (None, None, None, 1 512         conv3_block3_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_1_relu (Activation (None, None, None, 1 0           conv3_block3_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_2_conv (Conv2D)    (None, None, None, 1 147584      conv3_block3_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_2_bn (BatchNormali (None, None, None, 1 512         conv3_block3_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_2_relu (Activation (None, None, None, 1 0           conv3_block3_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_3_conv (Conv2D)    (None, None, None, 5 66048       conv3_block3_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_3_bn (BatchNormali (None, None, None, 5 2048        conv3_block3_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_add (Add)          (None, None, None, 5 0           conv3_block2_out[0][0]           \n",
      "                                                                 conv3_block3_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block3_out (Activation)   (None, None, None, 5 0           conv3_block3_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_1_conv (Conv2D)    (None, None, None, 1 65664       conv3_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_1_bn (BatchNormali (None, None, None, 1 512         conv3_block4_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_1_relu (Activation (None, None, None, 1 0           conv3_block4_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_2_conv (Conv2D)    (None, None, None, 1 147584      conv3_block4_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_2_bn (BatchNormali (None, None, None, 1 512         conv3_block4_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_2_relu (Activation (None, None, None, 1 0           conv3_block4_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_3_conv (Conv2D)    (None, None, None, 5 66048       conv3_block4_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_3_bn (BatchNormali (None, None, None, 5 2048        conv3_block4_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_add (Add)          (None, None, None, 5 0           conv3_block3_out[0][0]           \n",
      "                                                                 conv3_block4_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv3_block4_out (Activation)   (None, None, None, 5 0           conv3_block4_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_1_conv (Conv2D)    (None, None, None, 2 131328      conv3_block4_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_1_bn (BatchNormali (None, None, None, 2 1024        conv4_block1_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_1_relu (Activation (None, None, None, 2 0           conv4_block1_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_2_conv (Conv2D)    (None, None, None, 2 590080      conv4_block1_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_2_bn (BatchNormali (None, None, None, 2 1024        conv4_block1_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_2_relu (Activation (None, None, None, 2 0           conv4_block1_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_0_conv (Conv2D)    (None, None, None, 1 525312      conv3_block4_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_3_conv (Conv2D)    (None, None, None, 1 263168      conv4_block1_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_0_bn (BatchNormali (None, None, None, 1 4096        conv4_block1_0_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_3_bn (BatchNormali (None, None, None, 1 4096        conv4_block1_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_add (Add)          (None, None, None, 1 0           conv4_block1_0_bn[0][0]          \n",
      "                                                                 conv4_block1_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block1_out (Activation)   (None, None, None, 1 0           conv4_block1_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_1_conv (Conv2D)    (None, None, None, 2 262400      conv4_block1_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_1_bn (BatchNormali (None, None, None, 2 1024        conv4_block2_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_1_relu (Activation (None, None, None, 2 0           conv4_block2_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_2_conv (Conv2D)    (None, None, None, 2 590080      conv4_block2_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_2_bn (BatchNormali (None, None, None, 2 1024        conv4_block2_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_2_relu (Activation (None, None, None, 2 0           conv4_block2_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_3_conv (Conv2D)    (None, None, None, 1 263168      conv4_block2_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_3_bn (BatchNormali (None, None, None, 1 4096        conv4_block2_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_add (Add)          (None, None, None, 1 0           conv4_block1_out[0][0]           \n",
      "                                                                 conv4_block2_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block2_out (Activation)   (None, None, None, 1 0           conv4_block2_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_1_conv (Conv2D)    (None, None, None, 2 262400      conv4_block2_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_1_bn (BatchNormali (None, None, None, 2 1024        conv4_block3_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_1_relu (Activation (None, None, None, 2 0           conv4_block3_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_2_conv (Conv2D)    (None, None, None, 2 590080      conv4_block3_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_2_bn (BatchNormali (None, None, None, 2 1024        conv4_block3_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_2_relu (Activation (None, None, None, 2 0           conv4_block3_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_3_conv (Conv2D)    (None, None, None, 1 263168      conv4_block3_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_3_bn (BatchNormali (None, None, None, 1 4096        conv4_block3_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_add (Add)          (None, None, None, 1 0           conv4_block2_out[0][0]           \n",
      "                                                                 conv4_block3_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block3_out (Activation)   (None, None, None, 1 0           conv4_block3_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_1_conv (Conv2D)    (None, None, None, 2 262400      conv4_block3_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_1_bn (BatchNormali (None, None, None, 2 1024        conv4_block4_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_1_relu (Activation (None, None, None, 2 0           conv4_block4_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_2_conv (Conv2D)    (None, None, None, 2 590080      conv4_block4_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_2_bn (BatchNormali (None, None, None, 2 1024        conv4_block4_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_2_relu (Activation (None, None, None, 2 0           conv4_block4_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_3_conv (Conv2D)    (None, None, None, 1 263168      conv4_block4_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_3_bn (BatchNormali (None, None, None, 1 4096        conv4_block4_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_add (Add)          (None, None, None, 1 0           conv4_block3_out[0][0]           \n",
      "                                                                 conv4_block4_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block4_out (Activation)   (None, None, None, 1 0           conv4_block4_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_1_conv (Conv2D)    (None, None, None, 2 262400      conv4_block4_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_1_bn (BatchNormali (None, None, None, 2 1024        conv4_block5_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_1_relu (Activation (None, None, None, 2 0           conv4_block5_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_2_conv (Conv2D)    (None, None, None, 2 590080      conv4_block5_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_2_bn (BatchNormali (None, None, None, 2 1024        conv4_block5_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_2_relu (Activation (None, None, None, 2 0           conv4_block5_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_3_conv (Conv2D)    (None, None, None, 1 263168      conv4_block5_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_3_bn (BatchNormali (None, None, None, 1 4096        conv4_block5_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_add (Add)          (None, None, None, 1 0           conv4_block4_out[0][0]           \n",
      "                                                                 conv4_block5_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block5_out (Activation)   (None, None, None, 1 0           conv4_block5_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_1_conv (Conv2D)    (None, None, None, 2 262400      conv4_block5_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_1_bn (BatchNormali (None, None, None, 2 1024        conv4_block6_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_1_relu (Activation (None, None, None, 2 0           conv4_block6_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_2_conv (Conv2D)    (None, None, None, 2 590080      conv4_block6_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_2_bn (BatchNormali (None, None, None, 2 1024        conv4_block6_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_2_relu (Activation (None, None, None, 2 0           conv4_block6_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_3_conv (Conv2D)    (None, None, None, 1 263168      conv4_block6_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_3_bn (BatchNormali (None, None, None, 1 4096        conv4_block6_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_add (Add)          (None, None, None, 1 0           conv4_block5_out[0][0]           \n",
      "                                                                 conv4_block6_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv4_block6_out (Activation)   (None, None, None, 1 0           conv4_block6_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_1_conv (Conv2D)    (None, None, None, 5 524800      conv4_block6_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_1_bn (BatchNormali (None, None, None, 5 2048        conv5_block1_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_1_relu (Activation (None, None, None, 5 0           conv5_block1_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_2_conv (Conv2D)    (None, None, None, 5 2359808     conv5_block1_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_2_bn (BatchNormali (None, None, None, 5 2048        conv5_block1_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_2_relu (Activation (None, None, None, 5 0           conv5_block1_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_0_conv (Conv2D)    (None, None, None, 2 2099200     conv4_block6_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_3_conv (Conv2D)    (None, None, None, 2 1050624     conv5_block1_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_0_bn (BatchNormali (None, None, None, 2 8192        conv5_block1_0_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_3_bn (BatchNormali (None, None, None, 2 8192        conv5_block1_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_add (Add)          (None, None, None, 2 0           conv5_block1_0_bn[0][0]          \n",
      "                                                                 conv5_block1_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block1_out (Activation)   (None, None, None, 2 0           conv5_block1_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_1_conv (Conv2D)    (None, None, None, 5 1049088     conv5_block1_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_1_bn (BatchNormali (None, None, None, 5 2048        conv5_block2_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_1_relu (Activation (None, None, None, 5 0           conv5_block2_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_2_conv (Conv2D)    (None, None, None, 5 2359808     conv5_block2_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_2_bn (BatchNormali (None, None, None, 5 2048        conv5_block2_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_2_relu (Activation (None, None, None, 5 0           conv5_block2_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_3_conv (Conv2D)    (None, None, None, 2 1050624     conv5_block2_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_3_bn (BatchNormali (None, None, None, 2 8192        conv5_block2_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_add (Add)          (None, None, None, 2 0           conv5_block1_out[0][0]           \n",
      "                                                                 conv5_block2_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block2_out (Activation)   (None, None, None, 2 0           conv5_block2_add[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_conv (Conv2D)    (None, None, None, 5 1049088     conv5_block2_out[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_bn (BatchNormali (None, None, None, 5 2048        conv5_block3_1_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_1_relu (Activation (None, None, None, 5 0           conv5_block3_1_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_conv (Conv2D)    (None, None, None, 5 2359808     conv5_block3_1_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_bn (BatchNormali (None, None, None, 5 2048        conv5_block3_2_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_2_relu (Activation (None, None, None, 5 0           conv5_block3_2_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_3_conv (Conv2D)    (None, None, None, 2 1050624     conv5_block3_2_relu[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_3_bn (BatchNormali (None, None, None, 2 8192        conv5_block3_3_conv[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_add (Add)          (None, None, None, 2 0           conv5_block2_out[0][0]           \n",
      "                                                                 conv5_block3_3_bn[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "conv5_block3_out (Activation)   (None, None, None, 2 0           conv5_block3_add[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 23,587,712\n",
      "Trainable params: 23,534,592\n",
      "Non-trainable params: 53,120\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TensorShape([4, 7, 7, 2048])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 加载 ImageNet 预训练网络模型，并去掉最后一层\n",
    "resnet = keras.applications.ResNet50(weights='imagenet',include_top=False)\n",
    "resnet.summary()\n",
    "# 测试网络的输出\n",
    "x = tf.random.normal([4,224,224,3])\n",
    "# 获得子网络的输出\n",
    "out = resnet(x) \n",
    "out.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "从服务器下载模型结构和在ImageNet数据集上预训练好的网络参数。通过设置include_top参数为False，可以选择去掉ResNet50最后一层，此时网络的输出特征图大小为[b,7,7,2048]。  \n",
    "新建一个池化层(这里的池化层暂时可以理解为高、宽维度下采样的功能)，将特征从[b,7,7,2048]降维到[b, 2048]。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 2048)\n"
     ]
    }
   ],
   "source": [
    "# 新建池化层\n",
    "global_average_layer = layers.GlobalAveragePooling2D()\n",
    "# 利用上一层的输出作为本层的输入，测试其输出\n",
    "x = tf.random.normal([4,7,7,2048])\n",
    "# 池化层降维，形状由[4,7,7,2048]变为[4,1,1,2048],删减维度后变为[4,2048]\n",
    "out = global_average_layer(x)\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4, 100)\n"
     ]
    }
   ],
   "source": [
    "# 新建全连接层\n",
    "fc = layers.Dense(100)\n",
    "# 利用上一层的输出[4,2048]作为本层的输入，测试其输出\n",
    "x = tf.random.normal([4,2048])\n",
    "# 输出层的输出为样本属于 100 类别的概率分布\n",
    "out = fc(x) \n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在创建预训练的 ResNet50 特征子网络、 新建的池化层和全连接层后，我们重新利用Sequential 容器封装成一个新的网络。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "resnet50 (Model)             (None, None, None, 2048)  23587712  \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d (Gl (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 100)               204900    \n",
      "=================================================================\n",
      "Total params: 23,792,612\n",
      "Trainable params: 23,739,492\n",
      "Non-trainable params: 53,120\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# 重新包裹成我们的网络模型\n",
    "mynet = Sequential([resnet, global_average_layer, fc])\n",
    "mynet.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
